
(ns ca.uoit.molly.lucene-molly
  "Molly specific adaptors to Lucene functionality"
  (:use [ca.uoit.molly.lucene-helper]
        [clojure.contrib.str-utils]))

;
; Text processing
;
(defn ngram
  ([string n padding-char]
    (if (nil? string)
      '()
      (let [padding (apply str (repeat (dec n) padding-char))
          s (str padding (clojure.string/replace string " " padding) padding)]
        (map #(subs s % (+ n %)) (range (- (count s) (dec n)))))))
  ([string n]
    (ngram string n "$"))
  ([string]
    (ngram string 3 "$")))

(defn ngram-str
  [s]
  (str-join " " (ngram s)))

;
; Document conversion
;

(defn entity-to-doc
  "Converts a molly entity instance to a lucene document"
  [ent]
  (let [special-fields [{:f "__id__" :v (ent :id) :no-analyze true}
                        {:f "__entity__" :v (ent :entity) :no-analyze true}
                        {:f "__type__" :v "entity" :no-analyze true}
                        {:f "__all__" :v (str-join " " (vals (ent :attr)))}]
        attr-fields (for [[k v] (ent :attr)] {:f k :v v})]
    (apply mk-doc (into special-fields attr-fields))))

(defn is-special-field? [field-value]
  (let [field-name (-> 0 field-value)]
    (and (. field-name startsWith "__") (. field-name endsWith "__"))))

(defn field-to-attr [field-value]
  "Convert  [__xxx__ yyy] to [xxx yyy]"
  (let [[k v] field-value]
    [(keyword (. k replace "__" "")) v]))

(defn doc-to-entity
  "Converts a lucene document to a molly entity instance
   The structure of an entity instance is:
   {:__id__ <id>
    :__enttype__ <type>
    :__all__ <text>
    :attr (array-map :key val)}
  We assume that the doc has only field/value pairs."
  [doc]
  (let [all-pairs (for [f (. doc getFields)] 
                  [(.name f) (.stringValue f)])
        special-pairs (map field-to-attr (filter is-special-field? all-pairs))
        attr-pairs (map field-to-attr (filter (complement is-special-field?) all-pairs))]
    (assoc
      (apply hash-map (flatten special-pairs))
      :attr (apply array-map (flatten attr-pairs)))))

(defn attr-to-doc
  [v]
  (mk-doc {:f "entity" :v (v :entity) :no-index true}
          {:f "attr" :v (v :attr) :no-index true}
          {:f "value" :v (v :value) :no-index true}
          {:f "__all__" :v (ngram-str (v :value))}
          {:f "__type__" :v "attr"}))

(defn param-to-doc
  [p]
  (mk-doc {:f "param" :v (p 0) :no-index true}
          {:f "value" :v (p 1) :no-index true}
          {:f "__all__" :v (ngram-str (p 1))}
          {:f "__type__" :v "param"}))

(defn doc-to-value
  [doc]
  (let [all-pairs (for [f (. doc getFields)]
                   [(keyword (.name f)) (.stringValue f)])]
    (apply hash-map (flatten all-pairs))))

(defn- nodes-in-vein
  [vein]
  (for [v vein]
    {:entity (:entity v) :id (:id v)}))
(defn- nodes-in-denorm-group
  [denorm-grp]
  (set (flatten (for [vein denorm-grp] (nodes-in-vein vein)))))
(defn- paths-in-denorm-group
  [denorm-grp]
  (for [vein denorm-grp]
    (map #(identity {:entity (:entity %1) :id (:id %1)}) vein)))
(defn- params-in-vein
  "Gather the list of {:entity :id :key :value} of the parameters
  of the nodes in a vein"
  [vein]
  (apply clojure.set/union
    (for [v vein]
      (if (nil? (:parameters v))
        []
        (for [[key val] (:parameters v)]
          {:entity (:entity v) :id (:id v) :key key :value val})))))
(defn params-in-denorm-group
  [denorm-grp]
  (apply clojure.set/union
    (for [vein denorm-grp]
      (params-in-vein vein))))
(defn encode-node
  [n]
  (format "%s/%s" (:entity n) (. (str (:id n)) replace " " "&nbsp;")))
(defn- decode-node
  [n-str]
  (let [[ent id] (.split n-str "/" 2)]
    {:entity ent :id (. id replace "&nbsp;" " ")}))
(defn- nodes-to-field
  [nodes]
  {:f "__nodes__"
   :v (str-join " " (map encode-node nodes))})
(defn- path-to-field
  [path]
  {:f "__path__"
   :v (str-join " " (map encode-node path))
   :no-index true})
(defn- param-to-field
  [param]
  {:f "__param__"
   :v (format "%s %s %s" (encode-node param) (:key param) (:value param))})

(defn denorm-group-to-doc
  "denormalized group is the tree encoded as a list of veins.
  Each vertex in a vein is of the form {:entity ent :id id :param hash-map}.
  We encode the denormalized group into a document consisting of fields:
  entity-group: ...
  nodes/indexed: ent/id, ...
  path: ent/id ent/id ...
  path: ent/id ent/id ...
  ...
  param: ent/id k v
  param: ent/id k v
  "
  [grp-def denorm-grp]
  (let [nodes (nodes-in-denorm-group denorm-grp)
        params (params-in-denorm-group denorm-grp)
        paths (paths-in-denorm-group denorm-grp)]
    (do
;      (println "nodes=" nodes)
;      (println "params=" params)
;      (println "paths=" paths)
      (apply mk-doc (cons (nodes-to-field nodes)
        (clojure.set/union
          (map param-to-field params)
          (map path-to-field paths)))))))

;
; Document conversion back to denorm group
;
(defn- doc-to-param-map
  [doc]
  (let [
    update-param (
      fn [param param-str]
       (let [[node-str key val] (.split param-str " " 3)
             node (decode-node node-str)]
         (assoc-in param [node key] val)))]
    (loop [values (. doc getValues "__param__")
           param {}]
      (if (empty? values)
        param
        (recur (rest values) (update-param param (first values)))))))

(defn doc-to-denorm-group
  [doc]
  (let [param (doc-to-param-map doc)]
    (for [path-str (. doc getValues "__path__")
          :let [nodes (map decode-node (. path-str split " "))]]
      (vec (for [node nodes]
             (assoc node :parameters (param node)))))))

;
; Searching
;
(defn- analyze-search-apply
  [{:keys [idx query-str topk analyzer f]}]
  (let [tokens (analyzer query-str)
        query (apply mk-query (for [tok tokens] {:f "__all__" :v tok :or true}))]
    (map f (index-search idx query topk))))

(defn search-values
  [idx query-str topk]
  (analyze-search-apply {:idx idx :query-str query-str :topk topk
                         :analyzer ngram :f doc-to-value}))

(defn search-entities
  "DEFUNCT: Search for entities"
  [idx query-str topk]
  (let [a (fn [s] (analyze-str *default-analyzer* s))]
    (analyze-search-apply {:idx idx :query-str query-str :topk topk
                           :analyzer a :f doc-to-entity})))

(defn constraint-to-query-def
  [{:keys [entity attr value op]}]
  (let [op (keyword op)]
    (cons 
      {:f "__entity__" :v entity :and true}
      (for [token (analyze-str *default-analyzer* value)]
        {:f attr :v token :and (= op :and) :not (= op :not)}))))
  
(defn search-entities-by-constraints
  "Finds one or more entities matching the specified values."
  [idx topk constraints]
  (let [query-defs (flatten (map constraint-to-query-def constraints))
        query (apply mk-query query-defs)]
    (map doc-to-entity (index-search idx query topk))))

(defn search-groups
  "DEFUNCT: Search for entity groups by nodes and/or parameters
  the args is of the form: [{:node} {:param}]
  "
  [idx topk & args]
  (let [query
    (apply mk-query
      (flatten
        (for [query args]
          (let [field (if (:entity query) "__nodes__" "__param__")
                phrase (if (:entity query) (:entity query) (:param query))]
            (for [token (analyze-str *default-analyzer* phrase)]
              {:f field :v token :and false})))))]
    (index-search idx query topk)))
